From: Jo-Philipp Wich Date: Fri, 28 Jun 2019 11:05:58 +0000 (+0200) Subject: treewide: untangle phase1 and phase2 masters X-Git-Tag: v1~104 X-Git-Url: http://git.openwrt.org/%22https:/collectd.org//%22/%22https:/collectd.org/%22?a=commitdiff_plain;h=aef8d4aa1e284e45400103fa3526e7a90d4b3690;p=buildbot.git treewide: untangle phase1 and phase2 masters - Move shared scripts into a common scripts/ directory - Move SSH Git clone key directly into the configuration - Update configuration examples Signed-off-by: Jo-Philipp Wich --- diff --git a/.gitignore b/.gitignore index ee72b86..6eb6b27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ * !.gitignore +!scripts +!scripts/* !phase[12] !phase[12]/* phase[12]/*/* diff --git a/phase1/ccache.sh b/phase1/ccache.sh deleted file mode 100755 index 8756bc2..0000000 --- a/phase1/ccache.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C - -mkdir -p "$HOME/.ccache" || exit 1 - -grep -sq max_size "$HOME/.ccache/ccache.conf" || \ - echo "max_size = 10.0G" >> "$HOME/.ccache/ccache.conf" || exit 1 - -grep -sq compiler_check "$HOME/.ccache/ccache.conf" || \ - echo "compiler_check = %compiler% -dumpmachine; %compiler% -dumpversion" >> "$HOME/.ccache/ccache.conf" || exit 1 - -for dir in $(make --no-print-directory val.TOOLCHAIN_DIR val.STAGING_DIR val.STAGING_DIR_HOST V=s | grep staging_dir/); do - if [ ! -L "$dir/ccache" ] || [ -L "$dir/ccache" -a ! -d "$dir/ccache" ]; then - mkdir -vp "$dir" || exit 1 - rm -vrf "$dir/ccache" || exit 1 - ln -vs "$HOME/.ccache" "$dir/ccache" || exit 1 - fi -done - -exit 0 diff --git a/phase1/config.ini.example b/phase1/config.ini.example index f1b677c..7032806 100644 --- a/phase1/config.ini.example +++ b/phase1/config.ini.example @@ -1,12 +1,19 @@ [general] -title = LEDE Project -title_url = http://lede-project.org/ -buildbot_url = http://phase1.builds.lede-project.org/ +title = OpenWrt Project +title_url = http://openwrt.org/ +buildbot_url = http://phase1.builds.openwrt.org/ homedir = . expire = 1209600 port = 9989 cc_version = le 4.9 git_ssh = true +git_ssh_key = -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAuCJwo6OmrRDxcGfsMgBhq0vdzp2ZIdqnedFH8u6tVYLt9WDU + ... + mHzkh8Uv4OAWTjiLGycbXa0/31hu9PCeNzYmjjrp8tcGjsiJJFxydgS+wc0i2UPV + nSI+JbmAAF9vw6gj2i+Hqx7UloRd0tEv/leX354T5lO06LMiNhvN9g== + -----END RSA PRIVATE KEY----- + [status] bind = tcp:8010:interface=127.0.0.1 @@ -21,7 +28,7 @@ nickname = example-builder password = example [repo] -url = https://git.lede-project.org/source.git +url = https://git.openwrt.org/openwrt/openwrt.git branch = master [rsync] diff --git a/phase1/dumpinfo.pl b/phase1/dumpinfo.pl deleted file mode 100755 index aa97f8d..0000000 --- a/phase1/dumpinfo.pl +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use Cwd; - -my (%targets, %architectures); - -$ENV{'TOPDIR'} = Cwd::getcwd(); - - -sub parse_targetinfo { - my ($target_dir, $subtarget) = @_; - - if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET='$subtarget' |") { - my ($target_name, $target_arch, @target_features); - while (defined(my $line = readline M)) { - chomp $line; - - if ($line =~ /^Target: (.+)$/) { - $target_name = $1; - } - elsif ($line =~ /^Target-Arch-Packages: (.+)$/) { - $target_arch = $1; - } - elsif ($line =~ /^Target-Features: (.+)$/) { - @target_features = split /\s+/, $1; - } - elsif ($line =~ /^@\@$/) { - if ($target_name && $target_arch && - !grep { $_ eq 'broken' or $_ eq 'source-only' } @target_features) { - $targets{$target_name} = $target_arch; - $architectures{$target_arch} ||= []; - push @{$architectures{$target_arch}}, $target_name; - } - - undef $target_name; - undef $target_arch; - @target_features = (); - } - } - close M; - } -} - -sub get_targetinfo { - foreach my $target_makefile (glob "target/linux/*/Makefile") { - my ($target_dir) = $target_makefile =~ m!^(.+)/Makefile$!; - my @subtargets; - - if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.FEATURES V=s 2>/dev/null |") { - if (defined(my $line = readline M)) { - chomp $line; - if (grep { $_ eq 'broken' or $_ eq 'source-only' } split /\s+/, $line) { - next; - } - } - } - - if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS V=s 2>/dev/null |") { - if (defined(my $line = readline M)) { - chomp $line; - @subtargets = split /\s+/, $line; - } - close M; - } - - push @subtargets, 'generic' if @subtargets == 0; - - foreach my $subtarget (@subtargets) { - parse_targetinfo($target_dir, $subtarget); - } - } -} - -if (@ARGV == 1 && $ARGV[0] eq 'targets') { - get_targetinfo(); - foreach my $target_name (sort keys %targets) { - printf "%s %s\n", $target_name, $targets{$target_name}; - } -} -elsif (@ARGV == 1 && $ARGV[0] eq 'architectures') { - get_targetinfo(); - foreach my $target_arch (sort keys %architectures) { - printf "%s %s\n", $target_arch, join ' ', @{$architectures{$target_arch}}; - } -} -else { - print "Usage: $0 targets\n"; - print "Usage: $0 architectures\n"; -} diff --git a/phase1/expire.sh b/phase1/expire.sh deleted file mode 100755 index e965f84..0000000 --- a/phase1/expire.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -max_lifetime="$1" - -tree_birth="$(date --reference=tree.timestamp +%s 2>/dev/null)" -tree_age="$(( $(date +%s) - ${tree_birth:-0} ))" - -if [ $max_lifetime -le 0 ]; then - echo "No tree expiry set." - -elif [ $tree_age -ge $max_lifetime ]; then - echo "The build tree reached its maximum lifetime, cleaning up." - find . -mindepth 1 -maxdepth 1 -print0 | xargs -r -0 rm -vrf | while read entry; do - printf "." - done - - mkdir build - - echo "" - echo "Writing new timestamp" - date +%s > tree.timestamp - -else - echo "The build tree is not expired." -fi - -exit 0 diff --git a/phase1/findbin.pl b/phase1/findbin.pl deleted file mode 100755 index 5c9716a..0000000 --- a/phase1/findbin.pl +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -sub vernum($) { - if ($_[0] =~ m!^((?:\d+\.)+\d+)$!) { - my ($maj, $min) = split /\./, $1; - return int($maj) * 256 + int($min); - } - - return 0; -} - -sub vercmp($$$) { - my ($op, $v1, $v2) = @_; - - if ($op eq 'lt') { return $v1 < $v2 } - elsif ($op eq 'le') { return $v1 <= $v2 } - elsif ($op eq 'gt') { return $v1 > $v2 } - elsif ($op eq 'ge') { return $v1 >= $v2 } - elsif ($op eq 'eq') { return $v1 == $v2 } - - return 0; -} - -sub findbin($$$) { - my ($basename, $compare, $maxvstr) = @_; - - my $lastversion = 0; - my $cmpversion = vernum($maxvstr); - my $prog = undef; - - foreach my $dir (split /:/, $ENV{'PATH'}) { - foreach my $bin (glob("$dir/$basename?*"), "$dir/$basename") { - if (-x $bin && open BIN, '-|', $bin, '--version') { - my $vers = 0; - my $line = readline(BIN) || ''; - - foreach my $token (split /\s+/, $line) { - $vers = vernum($token); - last if $vers > 0; - } - - if ($vers > 0 && (!$cmpversion || vercmp($compare, $vers, $cmpversion))) { - if ($vers > $lastversion) { - $lastversion = $vers; - $prog = $bin; - } - } - - close BIN; - } - } - } - - return $prog; -} - -my $bin = findbin($ARGV[0], $ARGV[1], $ARGV[2]); - -if (defined $bin) { - printf "%s\n", $bin; - exit 0; -} -else { - warn "Cannot find a $ARGV[0] command with version $ARGV[1] $ARGV[2]\n"; - exit 1; -} diff --git a/phase1/makebranch.sh b/phase1/makebranch.sh deleted file mode 100755 index e92c04c..0000000 --- a/phase1/makebranch.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -git_author="Release Management" -git_email="lede-dev@lists.infradead.org" - -base_url="http://downloads.lede-project.org/releases" - -[ -f "./feeds.conf.default" ] || { - echo "Please execute as ./${0##*/}" >&2 - exit 1 -} - -usage() { - { - echo "" - echo "Usage: $0 [-i] [-a ] [-e ] \\" - echo " [-u ] -n -v " - echo "" - echo "-i" - echo "Exit successfully if branch already exists" - echo "" - echo "-a Git author [$git_author]" - echo "Override the author name used for automated Git commits" - echo "" - echo "-e Git email [$git_email]" - echo "Override the email used for automated Git commits" - echo "" - echo "-u Download base url [$base_url]" - echo "Use the given URL as base for download repositories" - echo "" - exit 1 - } >&2 -} - -while getopts "a:e:iu:n:v:" opt; do - case "$opt" in - a) git_author="$OPTARG" ;; - e) git_email="$OPTARG" ;; - i) ignore_existing=1 ;; - u) base_url="${OPTARG%/}" ;; - n) codename="$OPTARG" ;; - v) - case "$OPTARG" in - [0-9]*.[0-9]*) - version="$(echo "$OPTARG" | cut -d. -f1-2)" - ;; - *) - echo "Unexpected version format: $OPTARG" >&2 - exit 1 - ;; - esac - ;; - \?) - echo "Unexpected option: -$OPTARG" >&2 - usage - ;; - :) - echo "Missing argument for option: -$OPTARG" >&2 - usage - ;; - esac -done - -[ -n "$codename" -a -n "$version" ] || usage - -if git rev-parse "lede-${version}^{tree}" >/dev/null 2>/dev/null; then - if [ -z "$ignore_existing" ]; then - echo "Branch lede-${version} already exists!" >&2 - exit 1 - fi - - exit 0 -fi - -revnum="$(./scripts/getver.sh)" -githash="$(git log --format=%h -1)" - -prev_branch="$(git symbolic-ref -q HEAD)" - -if [ "$prev_branch" != "refs/heads/master" ]; then - echo "Expecting current branch name to be \"master\"," \ - "but it is \"${prev_branch#refs/heads/}\" - aborting." - - exit 1 -fi - -export GIT_AUTHOR_NAME="$git_author" -export GIT_AUTHOR_EMAIL="$git_email" -export GIT_COMMITTER_NAME="$git_author" -export GIT_COMMITTER_EMAIL="$git_email" - -git checkout -b "lede-$version" - -while read type name url; do - case "$type" in - src-git) - case "$url" in - *^*|*\;*) : ;; - *) - ref="$(git ls-remote "$url" "lede-$version")" - - if [ -z "$ref" ]; then - echo "WARNING: Feed \"$name\" provides no" \ - "\"lede-$version\" branch - using master!" >&2 - else - url="$url;lede-$version" - fi - ;; - esac - echo "$type $name $url" - ;; - src-*) - echo "$type $name $url" - ;; - esac -done < feeds.conf.default > feeds.conf.branch && \ - mv feeds.conf.branch feeds.conf.default - -sed -e 's!^RELEASE:=.*!RELEASE:='"$codename"'!g' \ - -e 's!\(VERSION_NUMBER:=\$(if .*\),[^,]*)!\1,'"$version-SNAPSHOT"')!g' \ - -e 's!\(VERSION_REPO:=\$(if .*\),[^,]*)!\1,'"$base_url/$version-SNAPSHOT"')!g' \ - include/version.mk > include/version.branch && \ - mv include/version.branch include/version.mk - -sed -e 's!http://downloads.lede-project.org/[^"]*!'"$base_url/$version-SNAPSHOT"'!g' \ - package/base-files/image-config.in > package/base-files/image-config.branch && \ - mv package/base-files/image-config.branch package/base-files/image-config.in - -git commit -sm "LEDE v$version: set branch defaults" \ - feeds.conf.default \ - include/version.mk \ - package/base-files/image-config.in - -git --no-pager log -p -1 -git push origin "refs/heads/lede-$version:refs/heads/lede-$version" -git checkout "${prev_branch#refs/heads/}" diff --git a/phase1/master.cfg b/phase1/master.cfg index edb4907..7521011 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -94,6 +94,7 @@ c['logHorizon'] = 20 ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) +scripts_dir = os.path.abspath("../scripts") tree_expire = 0 other_builds = 0 cc_version = None @@ -102,6 +103,7 @@ cc_command = "gcc" cxx_command = "g++" git_ssh = False +git_ssh_key = None if ini.has_option("general", "expire"): tree_expire = ini.getint("general", "expire") @@ -117,6 +119,11 @@ if ini.has_option("general", "cc_version"): if ini.has_option("general", "git_ssh"): git_ssh = ini.getboolean("general", "git_ssh") +if ini.has_option("general", "git_ssh_key"): + git_ssh_key = ini.get("general", "git_ssh_key") +else: + git_ssh = False + repo_url = ini.get("repo", "url") repo_branch = "master" @@ -163,7 +170,7 @@ if not os.path.isdir(home_dir+'/source.git'): else: subprocess.call(["git", "pull"], cwd = home_dir+'/source.git') -findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'], +findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'], stdout = subprocess.PIPE, cwd = home_dir+'/source.git') while True: @@ -453,7 +460,7 @@ for target in targets: if cc_version is not None: factory.addStep(FileDownload( name = "dlfindbinpl", - mastersrc = "findbin.pl", + mastersrc = scripts_dir + '/findbin.pl', slavedest = "../findbin.pl", mode = 0755)) @@ -486,7 +493,7 @@ for target in targets: factory.addStep(FileDownload( name = "dlexpiresh", doStepIf = IsExpireRequested, - mastersrc = "expire.sh", + mastersrc = scripts_dir + '/expire.sh', slavedest = "../expire.sh", mode = 0755)) @@ -628,9 +635,9 @@ for target in targets: # Git SSH if git_ssh: - factory.addStep(FileDownload( + factory.addStep(StringDownload( name = "dlgitclonekey", - mastersrc = "git-clone.key", + s = git_ssh_key, slavedest = "../git-clone.key", mode = 0600, )) @@ -923,7 +930,7 @@ for target in targets: factory.addStep(MasterShellCommand( name = "signfiles", description = "Signing files", - command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment], + command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment], env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile}, haltOnFailure = True )) @@ -990,7 +997,7 @@ for target in targets: # build list of files to upload factory.addStep(FileDownload( name = "dlsha2rsyncpl", - mastersrc = "sha2rsync.pl", + mastersrc = scripts_dir + '/sha2rsync.pl', slavedest = "../sha2rsync.pl", mode = 0755, )) @@ -1004,7 +1011,7 @@ for target in targets: factory.addStep(FileDownload( name = "dlrsync.sh", - mastersrc = "rsync.sh", + mastersrc = scripts_dir + '/rsync.sh', slavedest = "../rsync.sh", mode = 0755 )) diff --git a/phase1/rsync.sh b/phase1/rsync.sh deleted file mode 100755 index 64f9576..0000000 --- a/phase1/rsync.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -x - -export LC_ALL=C - -PV=`which pv` -RSYNC=rsync - -if [[ -x $PV ]]; then - $RSYNC "$@" | $PV -t -i 60 -f -else - $RSYNC "$@" -fi diff --git a/phase1/sha2rsync.pl b/phase1/sha2rsync.pl deleted file mode 100755 index f71437b..0000000 --- a/phase1/sha2rsync.pl +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/perl -w - -# ./sha2rsync.pl - -# is the filename of sha256sums fetched from server -# is the filename of sha256sums generated locally -# is the filename of the list of files to upload - -# and are files formatted as follows: -#checksum *pathtofile - -# both files must be sorted based on pathtofile: the script performs -# in-place merge (O(n+m)) of both lists based on that assumption. -# and are parsed only once. - -# the script cannot currently handle any other type of input - -# the script will generate , a list of files suitable for -# using with "rsync --files-from=" - -# if doesn't exist, all files in are added to the -# upload list. -# if exists, the files are added if: -# - they're not present in -# - they're present in AND their checksums differ - -# the script will clobber - -use strict; -use warnings; -use integer; - -die ("wrong number of arguments!") if ($#ARGV+1 != 3); - -my $shapat = qr/^(\w+) \*(.+)$/; - -my $rlist = $ARGV[0]; -my $llist = $ARGV[1]; -my $torsync = $ARGV[2]; - -my $rlist_fh = undef; -my $llist_fh = undef; -my $torsync_fh = undef; - -open($torsync_fh, ">", $torsync) or die("can't create output file!"); -open($llist_fh, "<", $llist) or die("can't read local list!"); -open($rlist_fh, "<", $rlist); - -my $lline = readline($llist_fh); -my $rline = readline($rlist_fh); - - -MAINLOOP: while () { - # run this loop as long as we have content from both rlist and llist - last (MAINLOOP) unless (defined($lline) && defined($rline)); - - chomp($lline); - my ($lcsum, $lfname) = $lline =~ $shapat; - - chomp($rline); - my ($rcsum, $rfname) = $rline =~ $shapat; - - # compare current remote and local filenames - my $rtlcmp = ($rfname cmp $lfname); - - while ($rtlcmp < 0) { # remote fname is before current local fname: remote file doesn't exist localy - $rline = readline($rlist_fh); - next (MAINLOOP); - } - - while ($rtlcmp > 0) { # remote fname is after current local fname: local file doesn't exist remotely - add_file($lfname); # add lfname to upload list - $lline = readline($llist_fh); - next (MAINLOOP); - } - - # if we end here, rtlcmp == 0: fnames matched, the file exist localy and remotely - - # fetch next line of both streams for the next iteration - $lline = readline($llist_fh); - $rline = readline($rlist_fh); - - # and skip if csums match - next (MAINLOOP) if ($lcsum eq $rcsum); - - # otherwise add the file as it's different - add_file($lfname); -} - -# deal with remainder of llist if any -while (defined($lline)) { - chomp($lline); - my ($lcsum, $lfname) = $lline =~ $shapat; - add_file($lfname); - $lline = readline($llist_fh); -} - -# unconditionally add some mandatory files to rsynclist -# add them last so they're transferred last: if everything else transferred correctly -add_file("packages/Packages.asc"); -add_file("sha256sums.asc"); -add_file("sha256sums"); - -exit (0); - -sub add_file { - my $fname = shift; - print $torsync_fh "$fname\n"; -} diff --git a/phase1/signall.sh b/phase1/signall.sh deleted file mode 100755 index c2575c5..0000000 --- a/phase1/signall.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -tarball="$1" -keyid="$2" -comment="$3" - -tmpdir="signall.$$" -tarball="$(readlink -f "$tarball")" - -finish() { rm -rf "$tmpdir"; exit $1; } - -trap "finish 255" HUP INT TERM - -if [ ! -f "$tarball" ]; then - echo "Usage: [GNUPGHOME=... [PASSFILE=...]] $0 [ []]" - finish 1 -fi - -umask 022 - -mkdir "$tmpdir" || finish 2 -tar -C "$tmpdir/" -xzf "$tarball" || finish 3 - -loopback="" - -case "$(gpg --version | head -n1)" in - *\ 2.*) loopback=1 ;; -esac - -find "$tmpdir/" -type f -not -name "*.asc" -exec gpg \ - --no-version --batch --yes -a -b \ - ${loopback:+--pinentry-mode loopback --no-tty --passphrase-fd 0} \ - ${keyid:+-u "$keyid"} \ - ${comment:+--comment="$comment"} \ - ${GNUPGHOME:+--homedir "$GNUPGHOME"} \ - ${PASSFILE:+--passphrase-file "$PASSFILE"} \ - -o "{}.asc" "{}" \; || finish 4 - -tar -C "$tmpdir/" -czf "$tarball" . || finish 5 - -finish 0 diff --git a/phase2/config.ini.example b/phase2/config.ini.example index 2df80f8..991a835 100644 --- a/phase2/config.ini.example +++ b/phase2/config.ini.example @@ -1,17 +1,27 @@ [general] -title = LEDE Project -title_url = http://lede-project.org/ -buildbot_url = http://phase2.builds.lede-project.org/ +title = OpenWrt Project +title_url = http://openwrt.org/ +buildbot_url = http://phase2.builds.openwrt.org/ homedir = ../phase1 port = 9990 persistent = false git_ssh = true +git_ssh_key = -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAuCJwo6OmrRDxcGfsMgBhq0vdzp2ZIdqnedFH8u6tVYLt9WDU + ... + mHzkh8Uv4OAWTjiLGycbXa0/31hu9PCeNzYmjjrp8tcGjsiJJFxydgS+wc0i2UPV + nSI+JbmAAF9vw6gj2i+Hqx7UloRd0tEv/leX354T5lO06LMiNhvN9g== + -----END RSA PRIVATE KEY----- [status] bind = tcp:8011:interface=127.0.0.1 user = example password = example +[repo] +url = https://git.openwrt.org/openwrt/openwrt.git +branch = master + [rsync] binary_url = user@example.org::upload-packages binary_password = example diff --git a/phase2/master.cfg b/phase2/master.cfg index 58f1b9f..bb0afaf 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -32,6 +32,7 @@ persistent = False other_builds = 0 tree_expire = 0 git_ssh = False +git_ssh_key = None if ini.has_option("general", "port"): slave_port = ini.getint("general", "port") @@ -48,6 +49,11 @@ if ini.has_option("general", "expire"): if ini.has_option("general", "git_ssh"): git_ssh = ini.getboolean("general", "git_ssh") +if ini.has_option("general", "git_ssh_key"): + git_ssh_key = ini.get("general", "git_ssh_key") +else: + git_ssh = False + c['slaves'] = [] max_builds = dict() @@ -76,6 +82,7 @@ c['logHorizon'] = 20 ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) +scripts_dir = os.path.abspath("../scripts") rsync_bin_url = ini.get("rsync", "binary_url") rsync_bin_key = ini.get("rsync", "binary_password") @@ -117,12 +124,23 @@ if ini.has_option("gpg", "comment"): if ini.has_option("gpg", "passfile"): gpg_passfile = ini.get("gpg", "passfile") +repo_url = ini.get("repo", "url") +repo_branch = "master" + +if ini.has_option("repo", "branch"): + repo_branch = ini.get("repo", "branch") + # find arches arches = [ ] archnames = [ ] -findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'], +if not os.path.isdir(home_dir+'/source.git'): + subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git']) +else: + subprocess.call(["git", "pull"], cwd = home_dir+'/source.git') + +findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'], stdout = subprocess.PIPE, cwd = home_dir+'/source.git') while True: @@ -266,7 +284,7 @@ for arch in arches: # expire tree if needed elif tree_expire > 0: factory.addStep(FileDownload( - mastersrc = home_dir+"/expire.sh", + mastersrc = scripts_dir + '/expire.sh', slavedest = "../expire.sh", mode = 0755)) @@ -333,7 +351,7 @@ for arch in arches: command = ["sh", "-c", "rm -f .config && make defconfig"])) factory.addStep(FileDownload( - mastersrc = home_dir+'/ccache.sh', + mastersrc = scripts_dir + '/ccache.sh', slavedest = 'sdk/ccache.sh', mode = 0755)) @@ -345,9 +363,9 @@ for arch in arches: haltOnFailure = True)) if git_ssh: - factory.addStep(FileDownload( + factory.addStep(StringDownload( name = "dlgitclonekey", - mastersrc = home_dir+"/git-clone.key", + s = git_ssh_key, slavedest = "../git-clone.key", mode = 0600)) @@ -429,7 +447,7 @@ for arch in arches: factory.addStep(MasterShellCommand( name = "signfiles", description = "Signing files", - command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment], + command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment], env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile}, haltOnFailure = True )) diff --git a/scripts/ccache.sh b/scripts/ccache.sh new file mode 100755 index 0000000..8756bc2 --- /dev/null +++ b/scripts/ccache.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +export LC_ALL=C + +mkdir -p "$HOME/.ccache" || exit 1 + +grep -sq max_size "$HOME/.ccache/ccache.conf" || \ + echo "max_size = 10.0G" >> "$HOME/.ccache/ccache.conf" || exit 1 + +grep -sq compiler_check "$HOME/.ccache/ccache.conf" || \ + echo "compiler_check = %compiler% -dumpmachine; %compiler% -dumpversion" >> "$HOME/.ccache/ccache.conf" || exit 1 + +for dir in $(make --no-print-directory val.TOOLCHAIN_DIR val.STAGING_DIR val.STAGING_DIR_HOST V=s | grep staging_dir/); do + if [ ! -L "$dir/ccache" ] || [ -L "$dir/ccache" -a ! -d "$dir/ccache" ]; then + mkdir -vp "$dir" || exit 1 + rm -vrf "$dir/ccache" || exit 1 + ln -vs "$HOME/.ccache" "$dir/ccache" || exit 1 + fi +done + +exit 0 diff --git a/scripts/dumpinfo.pl b/scripts/dumpinfo.pl new file mode 100755 index 0000000..aa97f8d --- /dev/null +++ b/scripts/dumpinfo.pl @@ -0,0 +1,91 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Cwd; + +my (%targets, %architectures); + +$ENV{'TOPDIR'} = Cwd::getcwd(); + + +sub parse_targetinfo { + my ($target_dir, $subtarget) = @_; + + if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET='$subtarget' |") { + my ($target_name, $target_arch, @target_features); + while (defined(my $line = readline M)) { + chomp $line; + + if ($line =~ /^Target: (.+)$/) { + $target_name = $1; + } + elsif ($line =~ /^Target-Arch-Packages: (.+)$/) { + $target_arch = $1; + } + elsif ($line =~ /^Target-Features: (.+)$/) { + @target_features = split /\s+/, $1; + } + elsif ($line =~ /^@\@$/) { + if ($target_name && $target_arch && + !grep { $_ eq 'broken' or $_ eq 'source-only' } @target_features) { + $targets{$target_name} = $target_arch; + $architectures{$target_arch} ||= []; + push @{$architectures{$target_arch}}, $target_name; + } + + undef $target_name; + undef $target_arch; + @target_features = (); + } + } + close M; + } +} + +sub get_targetinfo { + foreach my $target_makefile (glob "target/linux/*/Makefile") { + my ($target_dir) = $target_makefile =~ m!^(.+)/Makefile$!; + my @subtargets; + + if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.FEATURES V=s 2>/dev/null |") { + if (defined(my $line = readline M)) { + chomp $line; + if (grep { $_ eq 'broken' or $_ eq 'source-only' } split /\s+/, $line) { + next; + } + } + } + + if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS V=s 2>/dev/null |") { + if (defined(my $line = readline M)) { + chomp $line; + @subtargets = split /\s+/, $line; + } + close M; + } + + push @subtargets, 'generic' if @subtargets == 0; + + foreach my $subtarget (@subtargets) { + parse_targetinfo($target_dir, $subtarget); + } + } +} + +if (@ARGV == 1 && $ARGV[0] eq 'targets') { + get_targetinfo(); + foreach my $target_name (sort keys %targets) { + printf "%s %s\n", $target_name, $targets{$target_name}; + } +} +elsif (@ARGV == 1 && $ARGV[0] eq 'architectures') { + get_targetinfo(); + foreach my $target_arch (sort keys %architectures) { + printf "%s %s\n", $target_arch, join ' ', @{$architectures{$target_arch}}; + } +} +else { + print "Usage: $0 targets\n"; + print "Usage: $0 architectures\n"; +} diff --git a/scripts/expire.sh b/scripts/expire.sh new file mode 100755 index 0000000..e965f84 --- /dev/null +++ b/scripts/expire.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +max_lifetime="$1" + +tree_birth="$(date --reference=tree.timestamp +%s 2>/dev/null)" +tree_age="$(( $(date +%s) - ${tree_birth:-0} ))" + +if [ $max_lifetime -le 0 ]; then + echo "No tree expiry set." + +elif [ $tree_age -ge $max_lifetime ]; then + echo "The build tree reached its maximum lifetime, cleaning up." + find . -mindepth 1 -maxdepth 1 -print0 | xargs -r -0 rm -vrf | while read entry; do + printf "." + done + + mkdir build + + echo "" + echo "Writing new timestamp" + date +%s > tree.timestamp + +else + echo "The build tree is not expired." +fi + +exit 0 diff --git a/scripts/findbin.pl b/scripts/findbin.pl new file mode 100755 index 0000000..5c9716a --- /dev/null +++ b/scripts/findbin.pl @@ -0,0 +1,69 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +sub vernum($) { + if ($_[0] =~ m!^((?:\d+\.)+\d+)$!) { + my ($maj, $min) = split /\./, $1; + return int($maj) * 256 + int($min); + } + + return 0; +} + +sub vercmp($$$) { + my ($op, $v1, $v2) = @_; + + if ($op eq 'lt') { return $v1 < $v2 } + elsif ($op eq 'le') { return $v1 <= $v2 } + elsif ($op eq 'gt') { return $v1 > $v2 } + elsif ($op eq 'ge') { return $v1 >= $v2 } + elsif ($op eq 'eq') { return $v1 == $v2 } + + return 0; +} + +sub findbin($$$) { + my ($basename, $compare, $maxvstr) = @_; + + my $lastversion = 0; + my $cmpversion = vernum($maxvstr); + my $prog = undef; + + foreach my $dir (split /:/, $ENV{'PATH'}) { + foreach my $bin (glob("$dir/$basename?*"), "$dir/$basename") { + if (-x $bin && open BIN, '-|', $bin, '--version') { + my $vers = 0; + my $line = readline(BIN) || ''; + + foreach my $token (split /\s+/, $line) { + $vers = vernum($token); + last if $vers > 0; + } + + if ($vers > 0 && (!$cmpversion || vercmp($compare, $vers, $cmpversion))) { + if ($vers > $lastversion) { + $lastversion = $vers; + $prog = $bin; + } + } + + close BIN; + } + } + } + + return $prog; +} + +my $bin = findbin($ARGV[0], $ARGV[1], $ARGV[2]); + +if (defined $bin) { + printf "%s\n", $bin; + exit 0; +} +else { + warn "Cannot find a $ARGV[0] command with version $ARGV[1] $ARGV[2]\n"; + exit 1; +} diff --git a/scripts/makebranch.sh b/scripts/makebranch.sh new file mode 100755 index 0000000..e92c04c --- /dev/null +++ b/scripts/makebranch.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash + +git_author="Release Management" +git_email="lede-dev@lists.infradead.org" + +base_url="http://downloads.lede-project.org/releases" + +[ -f "./feeds.conf.default" ] || { + echo "Please execute as ./${0##*/}" >&2 + exit 1 +} + +usage() { + { + echo "" + echo "Usage: $0 [-i] [-a ] [-e ] \\" + echo " [-u ] -n -v " + echo "" + echo "-i" + echo "Exit successfully if branch already exists" + echo "" + echo "-a Git author [$git_author]" + echo "Override the author name used for automated Git commits" + echo "" + echo "-e Git email [$git_email]" + echo "Override the email used for automated Git commits" + echo "" + echo "-u Download base url [$base_url]" + echo "Use the given URL as base for download repositories" + echo "" + exit 1 + } >&2 +} + +while getopts "a:e:iu:n:v:" opt; do + case "$opt" in + a) git_author="$OPTARG" ;; + e) git_email="$OPTARG" ;; + i) ignore_existing=1 ;; + u) base_url="${OPTARG%/}" ;; + n) codename="$OPTARG" ;; + v) + case "$OPTARG" in + [0-9]*.[0-9]*) + version="$(echo "$OPTARG" | cut -d. -f1-2)" + ;; + *) + echo "Unexpected version format: $OPTARG" >&2 + exit 1 + ;; + esac + ;; + \?) + echo "Unexpected option: -$OPTARG" >&2 + usage + ;; + :) + echo "Missing argument for option: -$OPTARG" >&2 + usage + ;; + esac +done + +[ -n "$codename" -a -n "$version" ] || usage + +if git rev-parse "lede-${version}^{tree}" >/dev/null 2>/dev/null; then + if [ -z "$ignore_existing" ]; then + echo "Branch lede-${version} already exists!" >&2 + exit 1 + fi + + exit 0 +fi + +revnum="$(./scripts/getver.sh)" +githash="$(git log --format=%h -1)" + +prev_branch="$(git symbolic-ref -q HEAD)" + +if [ "$prev_branch" != "refs/heads/master" ]; then + echo "Expecting current branch name to be \"master\"," \ + "but it is \"${prev_branch#refs/heads/}\" - aborting." + + exit 1 +fi + +export GIT_AUTHOR_NAME="$git_author" +export GIT_AUTHOR_EMAIL="$git_email" +export GIT_COMMITTER_NAME="$git_author" +export GIT_COMMITTER_EMAIL="$git_email" + +git checkout -b "lede-$version" + +while read type name url; do + case "$type" in + src-git) + case "$url" in + *^*|*\;*) : ;; + *) + ref="$(git ls-remote "$url" "lede-$version")" + + if [ -z "$ref" ]; then + echo "WARNING: Feed \"$name\" provides no" \ + "\"lede-$version\" branch - using master!" >&2 + else + url="$url;lede-$version" + fi + ;; + esac + echo "$type $name $url" + ;; + src-*) + echo "$type $name $url" + ;; + esac +done < feeds.conf.default > feeds.conf.branch && \ + mv feeds.conf.branch feeds.conf.default + +sed -e 's!^RELEASE:=.*!RELEASE:='"$codename"'!g' \ + -e 's!\(VERSION_NUMBER:=\$(if .*\),[^,]*)!\1,'"$version-SNAPSHOT"')!g' \ + -e 's!\(VERSION_REPO:=\$(if .*\),[^,]*)!\1,'"$base_url/$version-SNAPSHOT"')!g' \ + include/version.mk > include/version.branch && \ + mv include/version.branch include/version.mk + +sed -e 's!http://downloads.lede-project.org/[^"]*!'"$base_url/$version-SNAPSHOT"'!g' \ + package/base-files/image-config.in > package/base-files/image-config.branch && \ + mv package/base-files/image-config.branch package/base-files/image-config.in + +git commit -sm "LEDE v$version: set branch defaults" \ + feeds.conf.default \ + include/version.mk \ + package/base-files/image-config.in + +git --no-pager log -p -1 +git push origin "refs/heads/lede-$version:refs/heads/lede-$version" +git checkout "${prev_branch#refs/heads/}" diff --git a/scripts/rsync.sh b/scripts/rsync.sh new file mode 100755 index 0000000..64f9576 --- /dev/null +++ b/scripts/rsync.sh @@ -0,0 +1,12 @@ +#!/bin/bash -x + +export LC_ALL=C + +PV=`which pv` +RSYNC=rsync + +if [[ -x $PV ]]; then + $RSYNC "$@" | $PV -t -i 60 -f +else + $RSYNC "$@" +fi diff --git a/scripts/sha2rsync.pl b/scripts/sha2rsync.pl new file mode 100755 index 0000000..f71437b --- /dev/null +++ b/scripts/sha2rsync.pl @@ -0,0 +1,109 @@ +#!/usr/bin/perl -w + +# ./sha2rsync.pl + +# is the filename of sha256sums fetched from server +# is the filename of sha256sums generated locally +# is the filename of the list of files to upload + +# and are files formatted as follows: +#checksum *pathtofile + +# both files must be sorted based on pathtofile: the script performs +# in-place merge (O(n+m)) of both lists based on that assumption. +# and are parsed only once. + +# the script cannot currently handle any other type of input + +# the script will generate , a list of files suitable for +# using with "rsync --files-from=" + +# if doesn't exist, all files in are added to the +# upload list. +# if exists, the files are added if: +# - they're not present in +# - they're present in AND their checksums differ + +# the script will clobber + +use strict; +use warnings; +use integer; + +die ("wrong number of arguments!") if ($#ARGV+1 != 3); + +my $shapat = qr/^(\w+) \*(.+)$/; + +my $rlist = $ARGV[0]; +my $llist = $ARGV[1]; +my $torsync = $ARGV[2]; + +my $rlist_fh = undef; +my $llist_fh = undef; +my $torsync_fh = undef; + +open($torsync_fh, ">", $torsync) or die("can't create output file!"); +open($llist_fh, "<", $llist) or die("can't read local list!"); +open($rlist_fh, "<", $rlist); + +my $lline = readline($llist_fh); +my $rline = readline($rlist_fh); + + +MAINLOOP: while () { + # run this loop as long as we have content from both rlist and llist + last (MAINLOOP) unless (defined($lline) && defined($rline)); + + chomp($lline); + my ($lcsum, $lfname) = $lline =~ $shapat; + + chomp($rline); + my ($rcsum, $rfname) = $rline =~ $shapat; + + # compare current remote and local filenames + my $rtlcmp = ($rfname cmp $lfname); + + while ($rtlcmp < 0) { # remote fname is before current local fname: remote file doesn't exist localy + $rline = readline($rlist_fh); + next (MAINLOOP); + } + + while ($rtlcmp > 0) { # remote fname is after current local fname: local file doesn't exist remotely + add_file($lfname); # add lfname to upload list + $lline = readline($llist_fh); + next (MAINLOOP); + } + + # if we end here, rtlcmp == 0: fnames matched, the file exist localy and remotely + + # fetch next line of both streams for the next iteration + $lline = readline($llist_fh); + $rline = readline($rlist_fh); + + # and skip if csums match + next (MAINLOOP) if ($lcsum eq $rcsum); + + # otherwise add the file as it's different + add_file($lfname); +} + +# deal with remainder of llist if any +while (defined($lline)) { + chomp($lline); + my ($lcsum, $lfname) = $lline =~ $shapat; + add_file($lfname); + $lline = readline($llist_fh); +} + +# unconditionally add some mandatory files to rsynclist +# add them last so they're transferred last: if everything else transferred correctly +add_file("packages/Packages.asc"); +add_file("sha256sums.asc"); +add_file("sha256sums"); + +exit (0); + +sub add_file { + my $fname = shift; + print $torsync_fh "$fname\n"; +} diff --git a/scripts/signall.sh b/scripts/signall.sh new file mode 100755 index 0000000..c2575c5 --- /dev/null +++ b/scripts/signall.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +tarball="$1" +keyid="$2" +comment="$3" + +tmpdir="signall.$$" +tarball="$(readlink -f "$tarball")" + +finish() { rm -rf "$tmpdir"; exit $1; } + +trap "finish 255" HUP INT TERM + +if [ ! -f "$tarball" ]; then + echo "Usage: [GNUPGHOME=... [PASSFILE=...]] $0 [ []]" + finish 1 +fi + +umask 022 + +mkdir "$tmpdir" || finish 2 +tar -C "$tmpdir/" -xzf "$tarball" || finish 3 + +loopback="" + +case "$(gpg --version | head -n1)" in + *\ 2.*) loopback=1 ;; +esac + +find "$tmpdir/" -type f -not -name "*.asc" -exec gpg \ + --no-version --batch --yes -a -b \ + ${loopback:+--pinentry-mode loopback --no-tty --passphrase-fd 0} \ + ${keyid:+-u "$keyid"} \ + ${comment:+--comment="$comment"} \ + ${GNUPGHOME:+--homedir "$GNUPGHOME"} \ + ${PASSFILE:+--passphrase-file "$PASSFILE"} \ + -o "{}.asc" "{}" \; || finish 4 + +tar -C "$tmpdir/" -czf "$tarball" . || finish 5 + +finish 0